package TFIDF;
require Exporter;
our @ISA       = qw(Exporter);
our @EXPORT_OK = qw(html2text html2tf html2tfidf text2tf tf2tfidf get_tokens);
our $VERSION = 1.00;

use warnings;
use strict;
use HTML::Parser;
use Lingua::EN::StopWords qw(%StopWords);

sub html2text {
    my $html = shift;
    my $parser = HTML::Parser->new(api_version => 3, marked_sections => 1);
    $parser->ignore_elements(qw(script style));
    $parser->handler(text => [], '@{dtext}');
    $parser->parse($html);
    $parser->eof;
    return join(' ', @{$parser->handler('text')});
}

sub html2tf {
    my ($html, $idf_ref) = shift;
    return text2tf(html2text($html), $idf_ref);
}

sub html2tfidf {
    my ($html, $idf_ref, $total) = @_;
    my $tf_ref = html2tf($html, $idf_ref);
    return tf2tfidf($tf_ref, $idf_ref, $total);
}

sub text2tf {
    my ($text, $dict_ref) = @_;
    return undef unless defined $text;
    my @rawTokens = @{get_tokens($text)};
    my @tokens;
    if (defined $dict_ref) {
        @tokens = grep {exists $dict_ref->{$_}} @rawTokens;
    }
    else {
        @tokens = grep {not exists $StopWords{$_}} @rawTokens;
    }
    my $tf_ref = {};
    $tf_ref->{$_}++ for @tokens;
    return $tf_ref;
}

sub tf2tfidf {
    my ($tf_ref, $idf_ref, $total) = @_;
    my $norm2 = 0;
    while (my($token, $tf) = each %$tf_ref) {
        my $info = $idf_ref->{$token};
        if (not defined $info) {
            delete $tf_ref->{$token};
            next;
        }
        my $idf = $info->[1];
        my $weight = get_weight($tf, $idf, $total);
        $tf_ref->{$token} = $weight;
        $norm2 += $weight ** 2;
    }
    $norm2 = sqrt($norm2);
    my $tfidf = [];
    for my $token (sort keys %$tf_ref) {
        my $index = $idf_ref->{$token}[0];
        my $weight = sprintf("%.6f", $tf_ref->{$token} / $norm2);
        push @$tfidf, [$index, $weight];
    }
    return $tfidf;
}

sub get_tokens {
    my $text = shift;
    my @tokens =  lc($text) =~ /([a-z]+)/g;
    return \@tokens;
}

sub get_weight {
  my ($tf, $idf, $total) = @_;
  return $tf * log($total / $idf);
}
